Advanced Lane Finding Project¶
The goals / steps of this project are the following:
- Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
- Apply a distortion correction to raw images.
- Use color transforms, gradients, etc., to create a thresholded binary image.
- Apply a perspective transform to rectify binary image ("birds-eye view").
- Detect lane pixels and fit to find the lane boundary.
- Determine the curvature of the lane and vehicle position with respect to center.
- Warp the detected lane boundaries back onto the original image.
- Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position
1. Camera Calibration and Distortion Correction¶
- Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
- Apply a distortion correction to raw images.
In [7]:
import pickle
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os
import glob
class CameraCalibrator():
"""
Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
Apply a distortion correction to raw images
"""
def __init__(self):
self.mtx, self.dist = self.calculate_mtx_dist()
def calculate_mtx_dist(self):
# prepare object points
objp = np.zeros((9*6, 3), np.float32)
objp[:, :2] = np.mgrid[0:9, 0:6].T.reshape(-1, 2)
# arrays to store object points and image points
objpoints = []
imgpoints = []
# make a list of calibration images
images = glob.glob(os.path.join('camera_cal', "calibration*.jpg"))
for idx, fname in enumerate(images):
img = cv2.imread(fname)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# find the chessboard corners
ret, corners = cv2.findChessboardCorners(img, (9, 6), None)
# If found, draw corners
if ret == True:
objpoints.append(objp)
imgpoints.append(corners)
# do camera calibration under the object points and image points
image_size = (img.shape[1], img.shape[0])
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, image_size, None, None)
return mtx, dist
def undistort(self, image):
undist = cv2.undistort(image, self.mtx, self.dist, self.mtx)
return undist
In [5]:
# Display and store chessboard image
image = mpimg.imread("camera_cal/calibration1.jpg")
camera_calibrator = CameraCalibrator()
undistort = camera_calibrator.undistort(image)
f, (ax1, ax2) = plt.subplots(1, 2, figsize = (12, 4))
f.tight_layout()
ax1.imshow(image)
ax1.set_title("Original Image", fontsize = 12)
ax2.imshow(undistort)
ax2.set_title("Undistored Image", fontsize = 12)
plt.tight_layout()
plt.subplots_adjust(left = 0, right = 1, top = 0.9, bottom = 0.)
plt.suptitle("Camera Calibration and Distortion Correction", y = 1.0, fontsize=16)
plt.savefig("output_images/1_undistort.jpg", bbox_inches='tight')
In [6]:
# Display and store chessboard image
image = mpimg.imread("test_images/test3.jpg")
camera_calibrator = CameraCalibrator()
undistort = camera_calibrator.undistort(image)
f, (ax1, ax2) = plt.subplots(1, 2, figsize = (12, 4))
f.tight_layout()
ax1.imshow(image)
ax1.set_title("Original Image", fontsize = 12)
ax2.imshow(undistort)
ax2.set_title("Undistored Image", fontsize = 12)
plt.subplots_adjust(left = 0, right = 1, top = 0.9, bottom = 0.)
plt.suptitle("Distortion Correction - test3.jpg", y = 1.0, fontsize=16)
plt.savefig("output_images/1_undistort_test3.jpg", bbox_inches='tight')
2. Color, Gradient Magnitude and Direction Threshold to binary image¶
- Use color transforms, gradients, etc., to create a thresholded binary image.
In [8]:
import cv2
import numpy as np
class BinaryThresholder():
'''Create a binary threshold image for detecting lane lines'''
def combined_binary(self, img):
'''Create a binary threshold image and apply it to self.image'''
# Apply color thresh
color_binary = color_tresh(img, s_thresh = (120, 255), v_thresh = (50, 255))
# Sobel_x thresh
gradx = abs_sobel_tresh(img, orient = 'x', thresh = (20, 255))
grady = abs_sobel_tresh(img, orient = 'y', thresh = (25, 255))
# extract yellow
ylw = extract_yellow(img)
# combined thresholdy
combined_binary = np.zeros_like(color_binary)
combined_binary[((gradx == 1) & (grady == 1)) | (color_binary == 1) | (ylw == 255)] = 1
# Apply a mask to select intresting
binary = region_of_interest(combined_binary)
return binary
def color_tresh(img, s_thresh = (0, 255), v_thresh = (0, 255)):
"""
Apply a thresh to colors, return binary image.
"""
# Extract s color
hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
s = hls[:, :, 2]
s_binary = np.zeros_like(s)
s_binary[(s >= s_thresh[0]) & (s <= s_thresh[1])] = 1
# Extract v color
hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
v = hsv[:, :, 2]
v_binary = np.zeros_like(v)
v_binary[(v > v_thresh[0]) & (v <= v_thresh[1])] = 1
output = np.zeros_like(v)
output[(s_binary == 1) & (v_binary == 1)] = 1
return output
def abs_sobel_tresh(img, orient = 'x', sobel_kernel = 3, thresh = (0, 255)):
"""
Define a function that applies Sobel x or y, and then apply a thresh.
return a binary image.
"""
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
if orient == 'x':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0))
elif orient == 'y':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1))
scaled_sobel = np.uint8(255*abs_sobel / np.max(abs_sobel))
binary_output = np.zeros_like(scaled_sobel)
binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1
return binary_output
def extract_yellow(image):
"""
Extract the yellow color.
"""
hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
return cv2.inRange(hsv, (26, 43, 46), (34, 255, 255))
def region_of_interest(image):
'''
Applies an image mask. Only keeps the region of the image defined by the polygon formed from `vertices`.
The rest of the image is set to black
'''
img_size = image.shape[:2]
vertices = np.array([[(img_size[1] / 2 - 30, img_size[0] / 2 + 80), # top left
(0, img_size[0]), # bottom left
(img_size[1], img_size[0]), # bottom right
(img_size[1] / 2 + 80, img_size[0] / 2 + 80)]], dtype=np.int32) # bottom right
mask = np.zeros_like(image)
# Define a 3 channel or 1 channel color to fill the mask with depending on the input image
if len(image.shape) > 2:
channel_count = image.shape[2] # i.e. 3 or 4 depending on your image
ignore_mask_color = (255,) * channel_count
else:
ignore_mask_color = 255
# Fill pixels inside the polygon defined by "vertices" with the fill color (i.e. white)
cv2.fillPoly(mask, vertices, ignore_mask_color)
# Return the image only where mask pixels are nonzero
return cv2.bitwise_and(image, mask)
In [8]:
# Display and store output image
image = mpimg.imread("test_images/test3.jpg")
camera_calibrator = CameraCalibrator()
undistort = camera_calibrator.undistort(image)
# Apply threshold
binary = BinaryThresholder()
combined_binary = binary.combined_binary(undistort)
f, (ax1, ax2) = plt.subplots(1, 2, figsize = (12, 4))
f.tight_layout()
ax1.imshow(undistort)
ax1.set_title("Undistort Image", fontsize = 12)
ax2.imshow(combined_binary, cmap = 'gray')
ax2.set_title("Binary Image After Threshold", fontsize =12)
plt.subplots_adjust(left = 0, right = 1, top = 0.9, bottom = 0.)
plt.suptitle("Combined Threshold - test3.jpg", y = 1.0, fontsize=16)
plt.savefig("output_images/2_Combined_threshold.jpg", bbox_inches='tight')
3. Apply a perspective transform to rectify binary image ("birds-eye view").¶
In [9]:
import numpy as np
import cv2
class PerspectiveTransfromer():
"""
Compute the perspective transform, M, given source and destination points
"""
def __init__(self):
# The original image size
img_size = (720, 1280)
# source points
self.src = np.float32(
[[(img_size[1] / 2) - 60, img_size[0] / 2 + 100],
[((img_size[1] / 6) - 10), img_size[0]],
[(img_size[1] * 5 / 6) + 60, img_size[0]],
[(img_size[1] / 2 + 60), img_size[0] / 2 + 100]])
#destination points
self.dst = np.float32(
[[(img_size[1] / 4), 0],
[(img_size[1] / 4), img_size[0]],
[(img_size[1] * 3 / 4), img_size[0]],
[(img_size[1] * 3 / 4), 0]])
self.M = cv2.getPerspectiveTransform(self.src, self.dst)
self.Minv = cv2.getPerspectiveTransform(self.dst, self.src)
def warp(self, image):
warped = cv2.warpPerspective(image, self.M, (image.shape[1], image.shape[0]),
flags=cv2.INTER_LINEAR)
return warped
3.1 Apply Straight lines¶
In [10]:
# display and store output image
image = mpimg.imread("test_images/straight_lines1.jpg")
camera_calibrator = CameraCalibrator()
undistort = camera_calibrator.undistort(image)
# apply threshold
# binary = BinaryThresholder(undistort)
# binary_combined = binary.threshold_image()
# perspective transform
transformer = PerspectiveTransfromer()
warped = transformer.warp(undistort)
cv2.polylines(undistort, [transformer.src.astype(int)], True, (255, 0, 0),5)
cv2.polylines(warped, [transformer.dst.astype(int)], True, (255, 0, 0),5)
f, (ax1, ax2) = plt.subplots(1, 2, figsize = (12, 4))
f.tight_layout()
ax1.imshow(undistort)
ax1.set_title("Undistort Image with source points drawn", fontsize = 12)
ax2.imshow(warped, cmap='gray')
ax2.set_title("Warped result with dest. drawn", fontsize = 12)
plt.subplots_adjust(left = 0, right = 1, top = 0.9, bottom = 0.)
plt.suptitle("Perspective Transform - straight_lines1.jpg", y = 1.0, fontsize=16)
plt.savefig("output_images/3_PerspectiveTransform_straight.jpg", bbox_inches='tight')
3.2 Apply curve lines¶
In [11]:
# display and store output image
image = mpimg.imread("test_images/test3.jpg")
camera_calibrator = CameraCalibrator()
undistort = camera_calibrator.undistort(image)
#apply threshold
binary = BinaryThresholder()
combined_binary = binary.combined_binary(undistort)
# perspective transform
transformer = PerspectiveTransfromer()
warped = transformer.warp(combined_binary)
#cv2.polylines(undistort, [transformer.src.astype(int)], True, (255, 0, 0),5)
#cv2.polylines(warped, [transformer.dst.astype(int)], True, (255, 0, 0),5)
f, (ax1, ax2) = plt.subplots(1, 2, figsize = (12, 4))
f.tight_layout()
ax1.imshow(undistort)
ax1.set_title("Undistort Image with source points drawn", fontsize = 12)
ax2.imshow(warped, cmap='gray')
ax2.set_title("Warped result after perspective transform", fontsize = 12)
plt.subplots_adjust(left = 0, right = 1, top = 0.9, bottom = 0.)
plt.suptitle("PerspectiveTransform for Curve lines - test3.jpg", y = 1.0, fontsize=16)
plt.savefig("output_images/4_PerspectiveTransform_curve.jpg", bbox_inches='tight')
4. Detect lane pixels and fit to find the lane boundary and compute the boundary.¶
In [10]:
import numpy as np
import cv2
import matplotlib.pyplot as plt
# HYPERPARAMETERS
# Choose the number of sliding windows
NWINDOWS = 9
# Set the width of the windows +/- margin
MARGIN = 100
# Set minimum number of pixels found to recenter window
MINPIX = 50
# Define conversions in x and y from pixels space to meters
YM_PER_PIX = 30/720 # meters per pixel in y dimension
XM_PER_PIX = 3.7/700 # meters per pixel in x dimension
class Line():
"""
1. Find the best fit of left and right lane lines through sliding windows.
2. Once found the lane lines in one frame of video, and then search within a window around the previous detection.
3. Smooth the lane fit line though taking an average over n past measurements.
3. Once get a good measurement of the line positions in warped space,
and draw back on the road with curvature and car position
"""
def __init__(self, n_frame=1, smooth_factor = 50):
# Taking an average over 'smooth_factor' past measurement
self.smooth_factor = smooth_factor
# when n_frame = 1, search lanes through sliding windows.
# Otherwise, search around the previous measurements.
self.n_frame = n_frame
# x values of the last n fits of the line tp smooth the detected lines
self.recent_xfitted = []
# The x coordinates of detected pixels on left lane line
self.leftx = None
# The y coordinates of detected pixels on left lane line
self.lefty = None
# The x coordinates of detected pixels on right lane line
self.rightx = None
# The y coordinates of detected pixels on right lane line
self.righty = None
# Through above four avariables, we can get the x coordiante of the best fitted polynominal
self.best_left_fitx = None
self.best_right_fitx = None
def find_best_fitx(self, warped):
"""
Find the x coordinates of the best fitted polynominal about lane lines.
"""
# n_frame =1 means that we can get the best fitted x through sliding windows
if self.n_frame == 1:
# Using the function 'find_lane_pixels' that is the silding window method to
# get the pixels' coordinates of lane lines
self.leftx, self.lefty, self.rightx, self.righty, _ = find_lane_pixels(warped)
# Get the polynomial x coordinate
left_fitx, right_fitx, ploty = generate_fitx(warped, self.leftx, self.lefty, self.rightx, self.righty)
# Append the polynomial x coordinate to the list so that we can smooth the
# lane lines by averaging previous 15 frames
self.recent_xfitted.append((left_fitx, right_fitx))
# Find the best fitted x coordinates of lane lines
self.best_left_fitx = left_fitx
self.best_right_fitx = right_fitx
elif self.n_frame > 1:
# n_frame > 1 means that we can get the best fitted x through searching around priors.
self.leftx, self.lefty, self.rightx, self.righty = search_from_prior(warped, self.leftx, self.lefty, self.rightx, self.righty)
# Get the polynomial x coordinate
left_fitx, right_fitx, ploty = generate_fitx(warped, self.leftx, self.lefty, self.rightx, self.righty)
# Append the polynomial x coordinate to the list so that we can smooth the
# lane lines by averaging previous 15 frames
self.recent_xfitted.append((left_fitx, right_fitx))
# Average the measuremtnts of previous 15 frames as the current best values.
best_xfitted = np.average(self.recent_xfitted[-self.smooth_factor:], axis = 0)
# Find the best fitted x coordinates of lane lines
self.best_left_fitx = best_xfitted[0]
self.best_right_fitx = best_xfitted[1]
def plot_line_on_warp(self, warped):
"""
Draw the best fitted lines on warped image.
"""
# Define the y coordinates
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0])
# Create an image to draw on and an image to show the selection window
out_img = np.dstack((warped, warped, warped))*255
# Draw the left line using the cv2.polylines
left_curve = np.column_stack((self.best_left_fitx, ploty))
right_curve = np.column_stack((self.best_right_fitx, ploty))
cv2.polylines(out_img, np.int_([left_curve]), False, (255,0,0), 10)
cv2.polylines(out_img, np.int_([right_curve]), False, (0,0,255), 10)
return out_img
def inversed_warp(self, warped, undistort, Minv):
"""
Draw the best fitted lines on the warp image and project measurements back down onto the road
with the curvatures and car position to the lane center.
"""
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0])
# Create an image to draw the lines on
warp_zero = np.zeros_like(warped).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
# Recast the x and y points into usable format for cv2.fillPoly()
pts_left = np.array([np.transpose(np.vstack([self.best_left_fitx, ploty]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([self.best_right_fitx, ploty])))])
pts = np.hstack((pts_left, pts_right))
cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))
# Draw the left and right lane lines
left_curve = np.column_stack((self.best_left_fitx, ploty))
right_curve = np.column_stack((self.best_right_fitx, ploty))
cv2.polylines(color_warp, np.int_([left_curve]), False, (255,0,0), 10)
cv2.polylines(color_warp, np.int_([right_curve]), False, (0,0,255), 10)
# Project measurements back to original image space using inverse perspective matrix (Minv)
newwarp = cv2.warpPerspective(color_warp, Minv, (undistort.shape[1], undistort.shape[0]))
# Combine the result with the original image
result = cv2.addWeighted(undistort, 1, newwarp, 0.5, 0)
# Compute the position of the car
image_center_x = warped.shape[1] / 2
lane_center_x = (self.best_left_fitx[-1] + self.best_right_fitx[-1]) / 2
offset = (lane_center_x - image_center_x)*XM_PER_PIX
# Get the line curvature
left_curvature, right_curvature = self.getCurvature(warped)
# Add curvature and offset information on the final image
text = "{:.3f}km".format(left_curvature/1000.)
cv2.putText(result, "Left curvature = " + text, (50, 50), cv2.FONT_HERSHEY_DUPLEX, 2, (255, 255, 255), 3)
text = "{:.3f}km".format(right_curvature/1000.)
cv2.putText(result, "Right curvature = " + text, (50, 120), cv2.FONT_HERSHEY_DUPLEX, 2, (255, 255, 255), 3)
left_right = 'left' if offset < 0 else 'right'
cv2.putText(result, "Vehicle is " + str(abs(round(offset, 3))) + "m " + left_right + " of center",(50, 190),
cv2.FONT_HERSHEY_DUPLEX, 2, (255, 255, 255), 3)
# Visulization of image center and lane center
cv2.line(result, (int(lane_center_x), result.shape[0]), (int(lane_center_x), result.shape[0] - 20),
(255, 255, 255), 6)
cv2.line(result, (int(image_center_x), result.shape[0]), (int(image_center_x), result.shape[0] - 40),
(255, 255, 255), 6)
cv2.line(result, (int(lane_center_x), result.shape[0] - 2), (int(image_center_x), result.shape[0] - 2),
(255, 255,255), 6)
return result
def getCurvature(self, warped):
"""
Compute the real curvature of detected lane lines
"""
leftx = self.best_left_fitx[::-1] # Reverse to match top-to-bottom in y
rightx = self.best_right_fitx[::-1] # Reverse to match top-to-bottom in y
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0])
# Fit a second order polynomial to pixel positions in each fake lane line
left_fit_cr = np.polyfit(ploty * YM_PER_PIX, leftx* XM_PER_PIX, 2)
right_fit_cr = np.polyfit(ploty * YM_PER_PIX, rightx * XM_PER_PIX, 2)
# Define y-value where we want radius of curvature
# We'll choose the maximum y-value, corresponding to the bottom of the image
y_eval = np.max(ploty) * YM_PER_PIX
# Calculation of R_curve (radius of curvature)
left_curvature = (1 + (2*left_fit_cr[0]*y_eval + left_fit_cr[1])**2)**1.5 / np.abs(2*left_fit_cr[0])
right_curvature = (1 + (2*right_fit_cr[0]*y_eval + right_fit_cr[1])**2)**1.5 / np.abs(2*right_fit_cr[0])
return(left_curvature, right_curvature)
#######
# Below, functions which are used to the Line Class.
#######
def find_lane_pixels(warped):
"""
Implement Sliding Windows
"""
# Take a histogram of the bottom half of the image
histogram = np.sum(warped[warped.shape[0]//2:,:], axis=0)
# Create an output image to draw on and visualize the result
out_img = np.dstack((warped, warped, warped))
# Find the peak of the left and right halves of the histogram
# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0]//2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
# Set height of windows - based on nwindows above and image shape
window_height = np.int(warped.shape[0]//NWINDOWS)
# Identify the x and y positions of all nonzero pixels in the image
nonzero = warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Current positions to be updated later for each window in nwindows
leftx_current = leftx_base
rightx_current = rightx_base
# Create empty lists to receive left and right lane pixel indices
left_lane_inds = []
right_lane_inds = []
# Step through the windows one by one
for window in range(NWINDOWS):
win_y_low = warped.shape[0] - (window+1)*window_height
win_y_high = warped.shape[0] - window*window_height
### TO-DO: Find the four below boundaries of the window ###
win_xleft_low = leftx_current - MARGIN
win_xleft_high = leftx_current + MARGIN
win_xright_low = rightx_current - MARGIN
win_xright_high = rightx_current + MARGIN
# Draw the windows on the visualization image
cv2.rectangle(out_img,(win_xleft_low,win_y_low),
(win_xleft_high,win_y_high),(0,255,0), 5)
cv2.rectangle(out_img,(win_xright_low,win_y_low),
(win_xright_high,win_y_high),(0,255,0), 5)
#Identify the nonzero pixels in x and y within the window ###
good_left_inds = ((nonzeroy > win_y_low) & (nonzeroy < win_y_high) & (nonzerox > win_xleft_low) &
(nonzerox < win_xleft_high)).nonzero()[0]
good_right_inds = ((nonzeroy > win_y_low) & (nonzeroy < win_y_high) & (nonzerox > win_xright_low) &
(nonzerox < win_xright_high)).nonzero()[0]
# Append these indices to the lists
left_lane_inds.append(good_left_inds)
right_lane_inds.append(good_right_inds)
### If you found > minpix pixels, recenter next window
if good_left_inds.sum() > MINPIX:
leftx_current = int(nonzerox[good_left_inds].mean())
if good_right_inds.sum() > MINPIX:
rightx_current = int(nonzerox[good_right_inds].mean())
# Concatenate the arrays of indices (previously was a list of lists of pixels)
try:
left_lane_inds = np.concatenate(left_lane_inds)
right_lane_inds = np.concatenate(right_lane_inds)
except ValueError:
# Avoids an error if the above is not implemented fully
pass
# Extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
return leftx, lefty, rightx, righty, out_img
def fit_polynomial(warped):
# Find our lane pixels first
leftx, lefty, rightx, righty, out_img = find_lane_pixels(warped)
### Fit a second order polynomial to each using `np.polyfit` ###
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# Generate x and y values for plotting
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0] )
try:
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
except TypeError:
# Avoids an error if `left` and `right_fit` are still none or incorrect
print('The function failed to fit a line!')
left_fitx = 1*ploty**2 + 1*ploty
right_fitx = 1*ploty**2 + 1*ploty
## Visualization ##
# Colors in the left and right lane regions
out_img[lefty, leftx] = [255, 0, 0]
out_img[righty, rightx] = [0, 0, 255]
# Plots the left and right polynomials on the lane lines
curve1 = np.column_stack((left_fitx, ploty))
curve2 = np.column_stack((right_fitx, ploty))
cv2.polylines(out_img, np.int_([curve1, curve2]), False, (0,255,255), 5)
return out_img
def search_from_prior(warped, leftx, lefty, rightx, righty):
"""
Find lane pixles' coordinates of the the current frame through searching the previous measurements
"""
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# Grab activated pixels
nonzero = warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
### n the previous quiz, but change the windows to our new search area
left_lane_inds = (nonzerox > (left_fit[0] * nonzeroy ** 2 + left_fit[1] * nonzeroy +
left_fit[2] - MARGIN)) & (nonzerox < (left_fit[0] * nonzeroy ** 2 +
left_fit[1] * nonzeroy + left_fit[2] + MARGIN))
right_lane_inds = (nonzerox > (right_fit[0] * nonzeroy ** 2 + right_fit[1] * nonzeroy +
right_fit[2] - MARGIN)) & (nonzerox < (right_fit[0] * nonzeroy ** 2 +
right_fit[1] * nonzeroy + right_fit[2] + MARGIN))
# Again, extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
return leftx, lefty, rightx, righty
def generate_fitx(warped, leftx, lefty, rightx, righty):
#Fit a second order polynomial to each using `np.polyfit` ###
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# Generate x and y values for plotting
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0])
try:
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
except TypeError:
# Avoids an error if `left` and `right_fit` are still none or incorrect
print('The function failed to fit a line!')
left_fitx = 1*ploty**2 + 1*ploty
right_fitx = 1*ploty**2 + 1*ploty
return left_fitx, right_fitx, ploty
In [13]:
# display and store output image
image = mpimg.imread("test_images/test3.jpg")
camera_calibrator = CameraCalibrator()
undistort = camera_calibrator.undistort(image)
#apply threshold
binary = BinaryThresholder()
combined_binary = binary.combined_binary(undistort)
# perspective transform
transformer = PerspectiveTransfromer()
warped = transformer.warp(combined_binary)
# sliding window image
window_img = fit_polynomial(warped)
f, (ax1, ax2) = plt.subplots(1, 2, figsize = (12, 4))
f.tight_layout()
ax1.imshow(undistort)
ax1.set_title("udistort image", fontsize = 12)
ax2.imshow(window_img, cmap='gray')
ax2.set_title("window image", fontsize = 12)
plt.subplots_adjust(left = 0, right = 1, top = 0.9, bottom = 0.)
plt.suptitle("Sliding Windows to Detect Lane Pixels - test3.jpg", y = 1.0, fontsize=16)
plt.savefig("output_images/4_2_sliding_windows_test3.jpg", bbox_inches='tight')
5. Processing the test images¶
In [11]:
############
## Pipeline for image
############
## For the image, the pipeline should be search through slinding window method to find best line
## For video, best lines were found through sliding window on first frame.
## But, Simply search best line within a window around the previous frame results.
## So the pipeline for image and vedio was different.
# Instantiate the classes.
# The camera calibraton and perpective transform should be done once with all images.
calibrator = CameraCalibrator()
binary = BinaryThresholder()
transformer = PerspectiveTransfromer()
line = Line()
def precess_image(img):
# Undisort the image
undistort = calibrator.undistort(img)
#Apply threshold to the undistort image
combined_binary = binary.combined_binary(undistort)
# perspective transform to get the warped image
warped = transformer.warp(combined_binary)
# sliding window image
window_img = fit_polynomial(warped)
line.find_best_fitx(warped)
road_image = line.inversed_warp(warped, undistort, transformer.Minv)
return undistort, window_img, road_image
In [ ]:
In [23]:
test_images = glob.glob(os.path.join('test_images', "*.jpg"))
fig, axis = plt.subplots(len(test_images), 3, figsize = (15, 25))
for row, image_path in enumerate(test_images):
img = mpimg.imread(image_path)
undistort, window_img, road_img = precess_image(img)
# Display undistort image, searching window image and road image every row
axis[row, 0].imshow(undistort)
#axis[row, 0].axis("off")
axis[row, 0].set_ylabel(image_path.split("\\")[-1])
axis[row, 1].imshow(window_img)
axis[row, 1].axis("off")
axis[row, 2].imshow(road_img)
axis[row, 2].axis("off")
plt.tight_layout()
plt.suptitle("original image windows image road image", y = 1.005, fontsize=25)
plt.savefig("output_images/all_test_images.jpg", bbox_inches = 'tight')
plt.show()
6. Processing vedio¶
In [6]:
############
## Pipeline for image
############
## For the image, the pipeline should be search through slinding window method to find best line
## For video, best lines were found through sliding window on first frame.
## But, Simply search best line within a window around the previous frame results.
## So the pipeline for image and vedio was different.
# Instantiate the classes.
# The camera calibraton and perpective transform should be done once with all images.
calibrator = CameraCalibrator()
binary = BinaryThresholder()
transformer = PerspectiveTransfromer()
# When instantiate the Line Class here, n_frame must be equal to 1.
# This means the first frame makes use of the sliding window method.
# 'smooth_factor' means I use how many frames to smooth the current measurement.But it shoule not be too bigger, or
# it may accumulates the previous errors. Within 10, the result is good.
line = Line(n_frame=1, smooth_factor=5)
def process_frame(img):
# Undisort the image
undistort = calibrator.undistort(img)
#Apply threshold to the undistort image
combined_binary = binary.combined_binary(undistort)
# perspective transform to get the warped image
warped = transformer.warp(combined_binary)
# plot lines on warped image.
# This is different from the image pipeline
line.find_best_fitx(warped)
line_image = line.plot_line_on_warp(warped)
road_image = line.inversed_warp(warped, undistort, transformer.Minv)
result = plot_diagnostics(undistort, combined_binary, line_image, road_image)
# This can control the function find_best_fitx of Class Line to select to search best line from priors.
line.n_frame += 1
# print(len(line.recent_xfitted))
# print(line.n_frame)
return result
def plot_diagnostics(undst_img, thres_img, line_img, road_img):
'''Plot a number of images created by the pipeline into a single image'''
diagScreen = np.zeros((960, 1280, 3), dtype=np.uint8)
# Main screen
diagScreen[0:720, 0:1280] = road_img
# Four screens along the bottom
diagScreen[720:960, 0:426] = cv2.resize(undst_img, (426, 240), interpolation=cv2.INTER_AREA) # undistorted
color_thresh = np.dstack((thres_img, thres_img, thres_img)) * 255
diagScreen[720:960, 426:852] = cv2.resize(color_thresh, (426, 240), interpolation=cv2.INTER_AREA) # binary
# threshold image
diagScreen[720:960, 852:1278] = cv2.resize(line_img, (426, 240), interpolation=cv2.INTER_AREA) # birds-eye with best line
# view of the binary threshold image
return diagScreen
6.1 Process an image using the video pipeline¶
In [14]:
image = mpimg.imread("test_images/test3.jpg")
result = process_frame(image)
fig = plt.figure(figsize = (12, 12))
plt.imshow(result)
plt.title("Combined all measurements - test3.jpg", fontsize=16)
plt.savefig("output_images\5_Combined_all_test3.jpg")
6.2 Process the veido: project_video.mp4¶
In [7]:
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
output = 'project_video_output.mp4'
clip1 = VideoFileClip('project_video.mp4')
out_clip = clip1.fl_image(process_frame)
%time out_clip.write_videofile(output, audio=False)
6.3 compute my pipeline velocity¶
In [9]:
total_frames = line.n_frame
total_time = 371 # s
smooth_factor = total_frames / total_time # FPS
print("The velocity of my pipeline: {} FPS".format(round(smooth_factor, 1)))
In [ ]: